# -*- coding: utf-8 -*-
import scrapy

import urllib
import unicodedata


class LovdSpiderSpider(scrapy.Spider):
    name = 'lovd_spider'
    allowed_domains = ['databases.lovd.nl']
    base_url = 'https://databases.lovd.nl/shared/variants/'

    def __init__(self, **kwargs):
        self.gene = kwargs.get('gene', 'A2M')

    def start_requests(self):

        url = self.base_url + self.gene
        yield scrapy.Request(url, meta={'gene': self.gene, 'page': 1})

    def parse(self, response):
        self.logger.debug('\033[32m>>> crawling {gene} page {page}\033[0m'.format(**response.meta))
        tableid = 'table#viewlistTable_CustomVL_VOT_VOG_{}'.format(response.meta['gene'])
        table = response.css(tableid)

        if not response.meta.get('title'):
            response.meta['title'] = [unicodedata.normalize('NFKC', each.strip()) for each in table.css('thead tr th ::text').extract() if each.strip()]
            # response.meta['search_transcriptid'] = response.css('input[name="search_transcriptid"]').attrib['value']

        for tr in table.css('tr.data'):
            row = [unicodedata.normalize('NFKC', each.strip()) for each in tr.css('td ::text').extract() if each.strip()]
            yield dict(zip(response.meta['title'], row))

        last_page = response.css('.pagesplit_nav tr th')

        if last_page and last_page[-1].attrib.get('class') != 'inactive':
            response.meta['page'] += 1
            payload = {
                'id': response.meta['gene'],
                'viewlistid': 'CustomVL_VIEW_' + response.meta['gene'],
                'page': str(response.meta['page']),
                'page_size': '100',
                'object': 'Custom_ViewList',
                # 'search_transcriptid': response.meta['search_transcriptid'],
                'object_id': 'VariantOnTranscript,VariantOnGenome,Screening,Individual',
            }
            url = '{}{}?{}'.format(self.base_url, response.meta['gene'], urllib.urlencode(payload))
            print('>>> URL: ' + url)
            yield response.follow(url, meta=response.meta)
